import re
import requests
from bs4 import BeautifulSoup as bs
import csv
url='http://rankings.betteredu.net/qs/major/tech/2022/computer-science-information-systems.html'
html=requests.get(url)
html.encoding='utf-8'
# with open('testfile.txt','w',encoding='utf-8')as f:
#     f.write(html.text)
target_test_demo=re.findall('<div class="rks_qsshowct">(.*?)</tr>',html.text,re.S)
zifucuan=''+target_test_demo[0]
target_head = bs(zifucuan, 'html.parser')
head_list_demo=target_head.find_all('th')
rows=[]
head_list=[]
for item in head_list_demo:
    head_list.append(item.string)
rows.append(head_list)
message=re.findall('<tbody>(.*?)</tbody>',html.text,re.S)
# with open('testfile1.txt','w',encoding='utf-8')as f:
#     f.write(message[0])
# print(table_all)
x = bs(message[0], 'html.parser')
message_list=x.find_all('tr')
# print(message_list)
n=1
for item in message_list:
    xinxi_list=item.find_all('td')
    row_message = []
    for item1 in xinxi_list:
        row_message.append(item1.string)
    rows.append(row_message)
    # n+=1
    # if n==6:
    #     break
# print(rows)
f=open('testfile.csv','w',encoding='utf-8',newline='')
writer=csv.writer(f)
for line in rows:
    writer.writerow(line)
f.close()
